/**
 * @date 2016年1月6日 下午9:34:43
 * @version V1.0
 */
package Commons;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;

/**
 * @ClassName: get_wiki_dict
 * @Description: TODO
 * @author ffftzh
 * @date 2016年1月6日 下午9:34:43
 *
 */
public class get_wiki_dict {
    public static void main(String[] args) throws Exception {
        String[] path = {"www_2008"};
        for (String item : path) {
            process("data2/" + item + "/wordmap.txt", "wiki.en.text");
        }
    }

    public static HashSet<Integer> getInteger(String[] wordlist) {
        HashSet<Integer> result = new HashSet<Integer>();
        for (int i = 1; i < wordlist.length; i++) {

            result.add(Integer.parseInt(wordlist[i]));
        }
        return result;
    }

    public static void get_wordPairs(String wikidoc) throws IOException {
        BufferedWriter writer = IOUtil.getWriter(wikidoc + ".pairs");
        BufferedReader word_reader = IOUtil.getReader(wikidoc);
        HashMap<String, HashSet<Integer>> wordDocId = new HashMap<String, HashSet<Integer>>();
        ArrayList<String> wordmap = new ArrayList<String>();
        String line = null;
        while ((line = word_reader.readLine()) != null) {
            String[] lines = line.split("\\s+");
            wordDocId.put(lines[0], getInteger(lines));
            wordmap.add(lines[0]);
        }

        for (int i = 0; i < wordmap.size(); i++) {
            for (int j = i; j < wordmap.size(); j++) {
                String word1 = wordmap.get(i), word2 = wordmap.get(j);
                String itemString = word1 + "|" + word2;
                if (word1.compareTo(word2) > 0) {
                    itemString = word2 + "|" + word1;
                }
                writer.write(itemString + " " + getnums(wordDocId.get(word1), wordDocId.get(word2)));
            }
        }
        writer.close();
        word_reader.close();
    }

    private static int getnums(HashSet<Integer> hashSet, HashSet<Integer> hashSet2) {
        HashSet<Integer> oooHashSet = new HashSet<Integer>();
        oooHashSet.addAll(hashSet);
        oooHashSet.retainAll(hashSet2);
        return oooHashSet.size();
    }

    public static void process(String wordMapPath, String wikiPath) throws Exception {

        // TODO Auto-generated method stub

//		DsparseTM.DsparseTMs(Integer.valueOf(args[0]), Integer.valueOf(args[1]),args[2]);
        BufferedReader word_reader = IOUtil.getReader(wordMapPath);

        HashMap<String, HashSet<Integer>> word2Docs = new HashMap<String, HashSet<Integer>>();
        String word2id = word_reader.readLine();
        while (word2id != null) {
            word2Docs.put(word2id.split("\\s+")[0], new HashSet<Integer>());
            word2id = word_reader.readLine();
        }
        word_reader.close();

        BufferedReader wiki_reader = IOUtil.getReader(wikiPath, "utf-8");
        String line = wiki_reader.readLine();
        int num = 0;
        while (line != null) {
            String[] tokens = line.split("\\s+");
            for (String token : tokens) {
                if (word2Docs.containsKey(token)) {
                    word2Docs.get(token).add(num);
                }
            }
            num++;
            if (num % 10000 == 0) {
                System.out.println(num);
            }
            line = wiki_reader.readLine();
        }
        wiki_reader.close();

        BufferedWriter writer = IOUtil.getWriter(wordMapPath + ".dict", "utf-8");
        for (String word : word2Docs.keySet()) {
            writer.append(word + "\t");
            for (int docId : word2Docs.get(word)) {
                writer.append(docId + " ");
            }
            writer.newLine();
        }
        writer.flush();
        writer.close();
    }

}
